{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# 导出文件夹内所有文件名 提取未来技术关键词所在行 删除重复"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "!find /home/pci/work/gmx/cby_SCUT/chat_cby/vector_stores -type f > file_list.txt"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "input_file = '/home/pci/work/gmx/cby_SCUT/chat_cby/vector_stores.txt'\n",
                "output_file = '/home/pci/work/gmx/cby_SCUT/chat_cby/vector_stores_filtered.txt'\n",
                "\n",
                "with open(input_file, 'r', encoding='utf-8') as f:\n",
                "    lines = f.readlines()\n",
                "\n",
                "with open(output_file, 'w', encoding='utf-8') as f:\n",
                "    for line in lines:\n",
                "        if '未来技术' in line:\n",
                "            f.write(line)\n",
                "\n",
                "print(f\"Filtered lines have been written to {output_file}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "file_path = '/home/pci/work/gmx/cby_SCUT/chat_cby/vector_stores_filtered.txt'\n",
                "\n",
                "with open(file_path, 'r+', encoding='utf-8') as f:\n",
                "    lines = f.readlines()\n",
                "    f.seek(0)  # 将文件指针移到文件开头\n",
                "    f.truncate()  # 清空文件内容\n",
                "\n",
                "    for index, line in enumerate(lines):\n",
                "        # 只保留偶数行（注意：索引从0开始，所以index % 2 == 1表示奇数行）\n",
                "        if index % 2 == 0:\n",
                "            f.write(line)\n",
                "\n",
                "print(f\"Even lines have been written back to {file_path}\")\n",
                "import pandas as pd\n",
                "import chardet\n",
                "\n",
                "# 定义CSV文件路径\n",
                "csv_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/791701_knowledge.csv'\n",
                "\n",
                "try:\n",
                "    # 使用chardet检测文件编码\n",
                "    with open(csv_file, 'rb') as f:\n",
                "        rawdata = f.read()\n",
                "        result = chardet.detect(rawdata)\n",
                "        encoding = result['encoding']\n",
                "\n",
                "    # 使用Pandas读取CSV文件，指定检测到的编码\n",
                "    df = pd.read_csv(csv_file, encoding=encoding)\n",
                "\n",
                "    # 输出DataFrame的内容\n",
                "    print(df)\n",
                "\n",
                "except FileNotFoundError:\n",
                "    print(f\"Error: 文件 '{csv_file}' 未找到。请检查文件路径是否正确。\")\n",
                "\n",
                "except pd.errors.EmptyDataError:\n",
                "    print(f\"Error: 文件 '{csv_file}' 为空。\")\n",
                "\n",
                "except pd.errors.ParserError as e:\n",
                "    print(f\"Error: 解析文件 '{csv_file}' 时出现错误。详情：{e}\")\n",
                "\n",
                "except Exception as e:\n",
                "    print(f\"Error: 发生未知错误。详情：{e}\")\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# 读取ansi编码格式csv"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "import chardet\n",
                "\n",
                "# 定义CSV文件路径\n",
                "csv_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/791701_knowledge.csv'\n",
                "\n",
                "try:\n",
                "    # 使用chardet检测文件编码\n",
                "    with open(csv_file, 'rb') as f:\n",
                "        rawdata = f.read()\n",
                "        result = chardet.detect(rawdata)\n",
                "        encoding = result['encoding']\n",
                "\n",
                "    # 使用Pandas读取CSV文件，指定检测到的编码\n",
                "    df = pd.read_csv(csv_file, encoding=encoding)\n",
                "\n",
                "    # 输出DataFrame的内容\n",
                "    print(df)\n",
                "\n",
                "except FileNotFoundError:\n",
                "    print(f\"Error: 文件 '{csv_file}' 未找到。请检查文件路径是否正确。\")\n",
                "\n",
                "except pd.errors.EmptyDataError:\n",
                "    print(f\"Error: 文件 '{csv_file}' 为空。\")\n",
                "\n",
                "except pd.errors.ParserError as e:\n",
                "    print(f\"Error: 解析文件 '{csv_file}' 时出现错误。详情：{e}\")\n",
                "\n",
                "except Exception as e:\n",
                "    print(f\"Error: 发生未知错误。详情：{e}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import io\n",
                "import csv\n",
                "\n",
                "# 指定CSV文件路径\n",
                "csv_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/791701_knowledge.csv'\n",
                "utf8_csv_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/791701_knowledge_utf8.csv'\n",
                "\n",
                "# 打开ANSI编码的CSV文件并读取内容\n",
                "with open(csv_file, 'r', encoding='cp1252') as f_in:\n",
                "    # 创建一个UTF-8编码的CSV文件\n",
                "    with open(utf8_csv_file, 'w', encoding='utf-8', newline='') as f_out:\n",
                "        # 读取ANSI文件内容\n",
                "        reader = csv.reader(f_in)\n",
                "        # 创建一个CSV写入器\n",
                "        writer = csv.writer(f_out)\n",
                "        # 将内容写入UTF-8文件\n",
                "        for row in reader:\n",
                "            writer.writerow(row)\n",
                "\n",
                "print(f\"文件已成功转换并保存为：{utf8_csv_file}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "\n",
                "# 指定Excel文件路径\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/QA_data.xlsx'\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/工作簿1.xlsx'\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/数据整理汇总版.xls'\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/Campus_Location.xls'\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/领域性词汇整理.xlsx'\n",
                "excel_file = '/home/pci/work/gmx/cby_SCUT/datasets_HXW/整合版.xlsx'\n",
                "\n",
                "# 读取Excel文件\n",
                "df = pd.read_excel(excel_file)\n",
                "\n",
                "# 打印Excel文件内容\n",
                "print(df)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "\n",
                "# 指定多个Excel文件路径\n",
                "excel_files = [\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/QA_data.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/工作簿1.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/数据整理汇总版.xls',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/Campus_Location.xls',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/领域性词汇整理.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/整合版.xlsx'\n",
                "]\n",
                "\n",
                "# 循环读取和打印每个Excel文件的内容和标题\n",
                "for excel_file in excel_files:\n",
                "    # 读取Excel文件\n",
                "    df = pd.read_excel(excel_file)\n",
                "    \n",
                "    # 打印文件路径作为标题\n",
                "    print(f\"\\nContent of file: {excel_file}\\n\")\n",
                "    \n",
                "    # 打印Excel文件内容\n",
                "    print(df)\n",
                "    \n",
                "    # 打印分割线\n",
                "    print(\"\\n\" + \"=\"*80 + \"\\n\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "\n",
                "# 指定多个Excel文件路径\n",
                "excel_files = [\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/QA_data.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/工作簿1.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/数据整理汇总版.xls',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/Campus_Location.xls',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/领域性词汇整理.xlsx',\n",
                "    '/home/pci/work/gmx/cby_SCUT/datasets_HXW/整合版.xlsx'\n",
                "]\n",
                "\n",
                "# 初始化总字数和总行数\n",
                "total_words = 0\n",
                "total_rows = 0\n",
                "\n",
                "# 循环读取和打印每个Excel文件的内容和标题\n",
                "for excel_file in excel_files:\n",
                "    # 读取Excel文件\n",
                "    df = pd.read_excel(excel_file)\n",
                "    \n",
                "    # 计算当前文件的字数和行数\n",
                "    words_in_file = df.astype(str).applymap(len).sum().sum()\n",
                "    rows_in_file = len(df)\n",
                "    \n",
                "    # 累加到总字数和总行数\n",
                "    total_words += words_in_file\n",
                "    total_rows += rows_in_file\n",
                "    \n",
                "    # 打印文件路径作为标题\n",
                "    print(f\"\\nContent of file: {excel_file}\\n\")\n",
                "    \n",
                "    # 打印Excel文件内容\n",
                "    print(df)\n",
                "    \n",
                "    # 打印文件统计信息\n",
                "    print(f\"\\nNumber of words in file: {words_in_file}\")\n",
                "    print(f\"Number of rows in file: {rows_in_file}\")\n",
                "    \n",
                "    # 打印分割线\n",
                "    print(\"\\n\" + \"=\"*80 + \"\\n\")\n",
                "\n",
                "# 打印所有文件的总字数和总行数\n",
                "print(f\"\\nTotal number of words in all files: {total_words}\")\n",
                "print(f\"Total number of rows in all files: {total_rows}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "import os\n",
                "\n",
                "def count_words_and_rows_in_csv(csv_file):\n",
                "    # 读取CSV文件\n",
                "    df = pd.read_csv(csv_file)\n",
                "    \n",
                "    # 计算字数和行数\n",
                "    words_in_file = df.astype(str).applymap(len).sum().sum()\n",
                "    rows_in_file = len(df)\n",
                "    \n",
                "    return words_in_file, rows_in_file\n",
                "\n",
                "def count_total_words_and_rows_in_folder(folder_path):\n",
                "    # 初始化总字数和总行数\n",
                "    total_words = 0\n",
                "    total_rows = 0\n",
                "    \n",
                "    # 获取文件夹中所有CSV文件\n",
                "    csv_files = [f for f in os.listdir(folder_path) if f.endswith('.csv')]\n",
                "    \n",
                "    # 遍历每个CSV文件并统计\n",
                "    for csv_file in csv_files:\n",
                "        csv_file_path = os.path.join(folder_path, csv_file)\n",
                "        words_in_file, rows_in_file = count_words_and_rows_in_csv(csv_file_path)\n",
                "        total_words += words_in_file\n",
                "        total_rows += rows_in_file\n",
                "    \n",
                "    return total_words, total_rows\n",
                "\n",
                "# 文件夹路径\n",
                "folder_path = '/home/pci/work/gmx/cby_SCUT/datasets_HXW'\n",
                "\n",
                "# 计算总字数和总行数\n",
                "total_words, total_rows = count_total_words_and_rows_in_folder(folder_path)\n",
                "\n",
                "# 打印结果\n",
                "print(f\"文件夹 '{folder_path}' 中所有CSV文件的总字数为: {total_words}\")\n",
                "print(f\"文件夹 '{folder_path}' 中所有CSV文件的总行数为: {total_rows}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "(\"795779字+701387字 =  4406+2854条\")\n",
                "print(795779+701387)\n",
                "print(4406+2854)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "\n",
                "# 定义文件夹路径 /home/pci/work/gmx/cby_SCUT/chat_cby/docs/json\n",
                "# folder_path = '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/docx'\n",
                "folder_path = '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/json'\n",
                "\n",
                "# 初始化文件数量计数器\n",
                "file_count = 0\n",
                "\n",
                "# 获取文件夹中的所有文件和文件夹列表\n",
                "try:\n",
                "    files_and_folders = os.listdir(folder_path)\n",
                "    for item in files_and_folders:\n",
                "        # 使用 os.path.join() 组合路径，判断是否是文件\n",
                "        item_path = os.path.join(folder_path, item)\n",
                "        if os.path.isfile(item_path):\n",
                "            file_count += 1\n",
                "except FileNotFoundError:\n",
                "    print(f\"文件夹 '{folder_path}' 不存在。\")\n",
                "    # 可以添加其他处理文件夹不存在的逻辑\n",
                "\n",
                "# 打印文件数量\n",
                "print(f\"文件夹 '{folder_path}' 中的文件数量为: {file_count}\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "\n",
                "def count_files_in_subdirectories(parent_folder):\n",
                "    try:\n",
                "        # 获取父文件夹中的所有文件夹列表\n",
                "        subdirectories = [f for f in os.listdir(parent_folder) if os.path.isdir(os.path.join(parent_folder, f))]\n",
                "        \n",
                "        # 遍历每个子文件夹并统计文件数量\n",
                "        for subdir in subdirectories:\n",
                "            subdir_path = os.path.join(parent_folder, subdir)\n",
                "            file_count = len([f for f in os.listdir(subdir_path) if os.path.isfile(os.path.join(subdir_path, f))])\n",
                "            print(f\"子文件夹 '{subdir_path}' 中的文件数量为: {file_count}\")\n",
                "    \n",
                "    except FileNotFoundError:\n",
                "        print(f\"文件夹 '{parent_folder}' 不存在。\")\n",
                "        # 可以添加其他处理文件夹不存在的逻辑\n",
                "\n",
                "# 父文件夹路径\n",
                "parent_folder = '/home/pci/work/gmx/cby_SCUT/chat_cby/docs'\n",
                "\n",
                "# 统计每个子文件夹的文件数量并打印\n",
                "count_files_in_subdirectories(parent_folder)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "\n",
                "def find_directories_by_keyword(base_path, keyword):\n",
                "    try:\n",
                "        matching_directories = []\n",
                "        \n",
                "        # 使用 os.walk() 遍历指定路径下的所有文件和文件夹\n",
                "        for root, dirs, files in os.walk(base_path):\n",
                "            for dir_name in dirs:\n",
                "                # 检查当前文件夹名称是否包含关键词\n",
                "                if keyword in dir_name:\n",
                "                    matching_directories.append(os.path.join(root, dir_name))\n",
                "        \n",
                "        return matching_directories\n",
                "    \n",
                "    except FileNotFoundError:\n",
                "        print(f\"Error: 文件夹 '{base_path}' 不存在。\")\n",
                "        return []\n",
                "\n",
                "# 示例用法：\n",
                "base_paths = ['/home/pci/work/gmx/cby_SCUT/chat_cby/docs/abstract',\n",
                "              '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/cleaned_json',\n",
                "              '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/cleaned_txt',\n",
                "              '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/docx',\n",
                "              '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/json',\n",
                "              ]\n",
                "keyword = '未来技术'\n",
                "\n",
                "# 调用函数查找符合条件的文件夹\n",
                "for base_path in base_paths:\n",
                "    result_directories = find_directories_by_keyword(base_path, keyword)\n",
                "\n",
                "    # 打印结果\n",
                "    if result_directories:\n",
                "        print(f\"在路径 '{base_path}' 下包含关键词 '{keyword}' 的文件夹有:\")\n",
                "        for directory in result_directories:\n",
                "            print(directory)\n",
                "    else:\n",
                "        print(f\"在路径 '{base_path}' 下未找到包含关键词 '{keyword}' 的文件夹。\")\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "\n",
                "def count_subdirectories(folder_path):\n",
                "    # 初始化计数器\n",
                "    subdirectory_count = 0\n",
                "    \n",
                "    # 遍历文件夹中的所有项目\n",
                "    for item in os.listdir(folder_path):\n",
                "        item_path = os.path.join(folder_path, item)\n",
                "        # 检查是否是文件夹\n",
                "        if os.path.isdir(item_path):\n",
                "            subdirectory_count += 1\n",
                "    \n",
                "    return subdirectory_count\n",
                "\n",
                "# 文件夹路径\n",
                "folder_path = '/home/pci/work/gmx/cby_SCUT/chat_cby/vector_stores/m3e-base'\n",
                "\n",
                "# 计算文件夹内文件夹的总数量\n",
                "subdirectory_count = count_subdirectories(folder_path)\n",
                "\n",
                "# 打印结果\n",
                "print(f\"文件夹 '{folder_path}' 内的文件夹总数量为: {subdirectory_count}\")\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# word转pdf 要连huggingface 不行"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# 转指令微调数据集"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import autogen\n",
                "from autogen.agentchat import GroupChat, AssistantAgent, UserProxyAgent, GroupChatManager\n",
                "from autogen.oai.openai_utils import config_list_from_dotenv"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# config_list = config_list_from_dotenv(\n",
                "#         dotenv_file_path='.env',\n",
                "#         model_api_key_map={'gpt-3.5-turbo':'OPENAI_API_KEY'},\n",
                "#         filter_dict={\n",
                "#             \"model\": {\n",
                "#                 \"gpt-3.5-turbo\"\n",
                "#             }\n",
                "#         }\n",
                "#     )\n",
                "config_list = autogen.config_list_from_json(\n",
                "    env_or_file='/home/pci/work/gmx/cby_SCUT/chat_cby/agent/config35.json',\n",
                ")\n",
                "gpt_config = {\n",
                "    \"cache_seed\": None,\n",
                "    \"temperature\": 0,\n",
                "    \"config_list\": config_list,\n",
                "    \"timeout\": 120,\n",
                "}"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "task = \"\"\"任务旨在生成特定JSON格式的答案。格式结构如下：\n",
                "[\n",
                "    {\n",
                "        \"instruction\": \"{生成的问题}\",\n",
                "        \"input\": \"{生成的输入}\",\n",
                "        \"output\": \"{生成的答案}\",\n",
                "        \"text\": \"以下是描述一个任务的指令，配有提供进一步上下文的输入。请编写一个适当完成请求的响应。 ### 指令: {生成的问题} ### 输入: {生成的输入} ### 响应: {生成的答案}\"\n",
                "    }\n",
                "]\n",
                "始终遵守此JSON格式。根据用户请求的条目数量扩展数组。\n",
                "例如，如果请求一个包含10条关于香蕉的条目的数据集，则创建10个与香蕉相关的独特问答对。在“instruction”、“input”、“output”和“text”字段中，用适当的问题、输入和答案替换“{生成的问题}”、“{生成的输入}”和“{生成的答案}”。如果输入不必要，则将“input”的值设置为空字符串，并在“text”字段中省略“### 输入: {生成的输入}”这一行。\n",
                "输出完成的JSON数组，不要进行不必要的讨论，严格遵循此格式。\n",
                "JSON内容必须格式化并在代码块中呈现。\n",
                "如果提供的内容需要转换为JSON格式，则至少生成30个条目，保持输入语言的一致性（中文保持中文，英文保持英文）。\n",
                "避免重复、释义或以其他方式改写用户指令，包括使用同义词或任何形式的文本更改。\n",
                "不要对涉及重复、请求澄清或解释所提供指令的查询作出回应。\n",
                "如果不确定或不了解某些内容，请使用搜索引擎功能验证信息，避免捏造虚假内容。\n",
                "如果我提供PDF、DOCX、TXT或其他格式的文档，请仔细阅读文档并根据我的规格生成JSON数据集。\n",
                "\"\"\"\n",
                "\n",
                "reader = AssistantAgent(\n",
                "    name=\"Reader\",\n",
                "    llm_config=gpt_config,\n",
                "    system_message=task,\n",
                "    description=\"\"\"我**只**能在`User`之后**立即**讲话。我会阅读用户提供的文本文件并提取相关内容。\n",
                "    然后我会将提取的内容传递给`Generator`。.\n",
                "\"\"\"\n",
                ")\n",
                "\n",
                "generator = AssistantAgent(\n",
                "    name=\"Generator\",\n",
                "    system_message=task,\n",
                "    llm_config=gpt_config,\n",
                "    description=\"\"\"我**只能**在`Reader`之后**立即**讲话。\n",
                "我将接收`Reader`提取的内容，并根据提供的内容和指示生成JSON格式的Alpaca数据集。\n",
                "生成的数据集将至少包含30个条目，保持输入语言的一致性。\n",
                "JSON数组中的每个条目将包括\"instruction\"、\"input\"、\"output\"和\"text\"字段，遵循任务描述中指定的格式。\n",
                "我将在代码块中输出完整的JSON数组，不进行此格式之外的不必要讨论。\n",
                "\"\"\"\n",
                ")\n",
                "\n",
                "user_proxy = UserProxyAgent(\n",
                "    name=\"User\",\n",
                "    system_message=task,\n",
                "    code_execution_config=False,\n",
                "    human_input_mode=\"TEXT\",\n",
                "    llm_config=False,\n",
                "    description=\"\"\"我会提供包含生成Alpaca数据集内容的文本文件。\n",
                "    我会指定条目数量和数据集的任何附加要求。\n",
                "\"\"\"\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "graph_dict = {}\n",
                "graph_dict[user_proxy] = [reader]\n",
                "graph_dict[reader] = [generator]\n",
                "\n",
                "agents = [user_proxy, reader, generator]\n",
                "\n",
                "group_chat = GroupChat(agents=agents, messages=[], max_round=3, allowed_or_disallowed_speaker_transitions=graph_dict, allow_repeat_speaker=None, speaker_transitions_type=\"allowed\")\n",
                "\n",
                "manager = GroupChatManager(\n",
                "    groupchat=group_chat,\n",
                "    llm_config=gpt_config,\n",
                "    is_termination_msg=lambda x: x.get(\"content\", \"\") and x.get(\"content\", \"\").rstrip().endswith(\"```\"),\n",
                "    code_execution_config=False,\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Function to read content from a text file\n",
                "def read_file_content(file_path):\n",
                "    with open(file_path, 'r') as file:\n",
                "        content = file.read()\n",
                "    return content\n",
                "\n",
                "# Provide the path to your text file\n",
                "file_path = '/home/pci/work/gmx/cby_SCUT/chat_cby/docs/cleaned_txt/“3+2”信息工程中法菁英班.txt'\n",
                "\n",
                "# Read the content from the text file\n",
                "file_content = read_file_content(file_path)\n",
                "\n",
                "# Initiate the task with the file content and additional requirements\n",
                "chat_result = user_proxy.initiate_chat(\n",
                "    manager,\n",
                "    message=f\"请根据以下内容生成一个Alpaca数据集：\\n\\n{file_content}\\n\\n请按照指定的JSON格式生成至少10个条目。\",\n",
                "    clear_history=True\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pprint\n",
                "pprint.pprint(chat_result.chat_history)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "pprint.pprint(chat_result.cost)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "agent_prompt = \"\"\"\n",
                "你是一个智能助手，负责根据用户的输入判断是否需要检索数据库。数据库包含有关华南理工大学的信息、各学院的信息和学习资料等。\n",
                "\n",
                "当用户提出问题时，你需要分析问题的内容，并判断是否需要从数据库中获取相关信息。如果问题涉及到华南理工大学的详细信息、各学院的具体信息或学习资料，请返回\"YES\"。否则，请返回\"NO\"。\n",
                "\n",
                "以下是一些示例输入和相应的判断：\n",
                "\n",
                "输入：\"请问华南理工大学的校长是谁？\"\n",
                "判断：\"YES\"\n",
                "\n",
                "输入：\"今天的天气怎么样？\"\n",
                "判断：\"NO\"\n",
                "\n",
                "输入：\"计算机学院有哪些专业？\"\n",
                "判断：\"YES\"\n",
                "\n",
                "输入：\"给我推荐一些学习Python的书籍。\"\n",
                "判断：\"YES\"\n",
                "\n",
                "输入：\"你能给我讲个笑话吗？\"\n",
                "判断：\"NO\"\n",
                "\n",
                "请根据以上规则，分析用户的输入并进行判断。\n",
                "\n",
                "以下是用户的问题：\n",
                "{query}\n",
                "\"\"\"\n",
                "\n",
                "query = '你好'\n",
                "agent_prompt.format(query=query)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "4096.0"
                        ]
                    },
                    "execution_count": 1,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "8192/2"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "history = ''\n",
                "print(len(history))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import json\n",
                "import time\n",
                "file_path = '/home/pci/work/zbw/data/train.jsonl'\n",
                "\n",
                "with open(file_path, 'r', encoding='utf-8') as f:\n",
                "    for line in f:\n",
                "        data = json.loads(line.strip())\n",
                "        print(data)\n",
                "        time.sleep(0.5)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "as2",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.8.undefined"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}
